{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def range_matrix(r, c):\n",
    "    return np.arange(r * c).reshape((r, c)) * 0.1 + 0.1\n",
    "\n",
    "\n",
    "input_len = 3\n",
    "num_classes = 3\n",
    "n, p = 0, 0\n",
    "hidden_size = 2  # size of hidden layer of neurons\n",
    "seq_length = 3  # number of steps to unroll the RNN for\n",
    "learning_rate = 1\n",
    "\n",
    "data_len = 50000\n",
    "x = np.arange(data_len) + 1\n",
    "\n",
    "ground_truth = [(x[i - 1] + x[i - 2]) % 3 for i in range(data_len)]\n",
    "\n",
    "# model parameters\n",
    "U = range_matrix(hidden_size, input_len)  # input to hidden\n",
    "W = range_matrix(hidden_size, hidden_size)  # hidden to hidden\n",
    "V = range_matrix(num_classes, hidden_size)  # hidden to output\n",
    "bs = np.zeros((hidden_size, 1))  # hidden bias\n",
    "bo = np.zeros((num_classes, 1))  # output bias\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_and_backprop(inputs, targets, hprev):\n",
    "  xs, hs, ys, ps = {}, {}, {}, {}\n",
    "  hs[-1] = np.copy(hprev)\n",
    "  loss = 0\n",
    "  # forward pass\n",
    "  for t in xrange(seq_length):\n",
    "    xs[t] = inputs[t:t+3].reshape(input_len, 1) # make a matrix(rank 2)\n",
    "    hs[t] = np.tanh(np.dot(U, xs[t]) + np.dot(W, hs[t - 1]) + bs) #计算hidden state。激活函数使用tanh\n",
    "    ys[t] = np.dot(V, hs[t]) + bo #计算output logits。注意这里没有激活函数，我们将在下一步计算softmax\n",
    "    \n",
    "    ps[t] = np.exp(ys[t]) / np.sum(np.exp(ys[t])) # softmax\n",
    "    \n",
    "    loss += np.mean(-np.sum(targets * np.log(ps[t]))) # 计算交叉熵\n",
    "    \n",
    "  #反向传播过程  \n",
    "  dU, dW, dV = np.zeros_like(U), np.zeros_like(W), np.zeros_like(V)\n",
    "  dbs, dbo = np.zeros_like(bs), np.zeros_like(bo)\n",
    "  dhnext = np.zeros_like(hs[0])\n",
    "\n",
    "  \n",
    "  for t in reversed(xrange(seq_length)):\n",
    "    dy = np.copy(ps[t])\n",
    "    dy[targets[t]] -= 1 # softmax-交叉熵delta： y-t\n",
    "    dV += np.outer(dy, hs[t]) #V-nabla\n",
    "    dbo = '''Fill your code HERE''' #bo-nabla\n",
    "    dh = '''Fill your code HERE''' # backprop into hidden-state\n",
    "    dhraw = (1 - hs[t] * hs[t]) * dh # tanh的导数是1-logits^2\n",
    "    dbs = '''Fill your code HERE''' #bs-nabla\n",
    "    dU = '''Fill your code HERE''' # U-nabla\n",
    "    if t>0:\n",
    "      dW = '''Fill your code HERE''' # W-nabla\n",
    "    dhnext = dhraw\n",
    "   \n",
    "    \n",
    "  return loss, dU, dW, dV, dbs, dbo, hs[seq_length-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "执行前向+反向传播5次（每次计算的time step为3）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "ufunc 'exp' not supported for the input types, and the inputs could not be safely coerced to any supported types according to the casting rule ''safe''",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-c57fd3ad9592>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     44\u001b[0m   \u001b[0minputs\u001b[0m \u001b[0;34m=\u001b[0m  \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mseq_length\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m   \u001b[0mtargets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mground_truth\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mseq_length\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m   \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdU\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdV\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdbs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdbo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhprev\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mforward_and_backprop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhprev\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m   \u001b[0;31m# perform parameter update with Adagrad\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m   for param, dparam in zip([U, W, V, bs, bo], \n",
      "\u001b[0;32m<ipython-input-17-c57fd3ad9592>\u001b[0m in \u001b[0;36mforward_and_backprop\u001b[0;34m(inputs, targets, hprev)\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mys\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'''Fill your code HERE'''\u001b[0m \u001b[0;31m#计算output logits。注意这里没有激活函数，我们将在下一步计算softmax\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m     \u001b[0mps\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mys\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mys\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# softmax\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0;31m#注意我们将所有time step的loss累加起来\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: ufunc 'exp' not supported for the input types, and the inputs could not be safely coerced to any supported types according to the casting rule ''safe''"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "for n in range(5):\n",
    "  # prepare inputs (we're sweeping from left to right in steps seq_length long)\n",
    "  if p+seq_length+1 >= len(x) or n == 0: \n",
    "    hprev = np.zeros((hidden_size,1)) # reset RNN memory\n",
    "    p = 2 # go from start of data\n",
    "  inputs =  x[p-2:p+seq_length]\n",
    "  targets = ground_truth[p:p+seq_length]\n",
    "  loss, dU, dW, dV, dbs, dbo, hprev = forward_and_backprop(inputs, targets, hprev)\n",
    "  # perform parameter update with Adagrad\n",
    "  for param, dparam in zip([U, W, V, bs, bo], \n",
    "                                [dU, dW, dV, dbs, dbo]):\n",
    "    param += -learning_rate * dparam #sgd\n",
    "\n",
    "  p += seq_length # move data pointer\n",
    "\n",
    "print('U:')\n",
    "print(U)\n",
    "print('W:')\n",
    "print(W)\n",
    "print('V:')\n",
    "print(V)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "如果一切正确，你应该看到如下的结果：\n",
    "```\n",
    "U:\n",
    "[[-0.24492589 -0.23727184 -0.2296178 ]\n",
    " [ 0.39838373  0.49675484  0.59512595]]\n",
    "W:\n",
    "[[ 0.0992239   0.19968422]\n",
    " [ 0.3000113   0.40001275]]\n",
    "V:\n",
    "[[ 0.37622149  0.920997  ]\n",
    " [ 0.39517001  0.81996845]\n",
    " [ 0.1286085  -0.54096546]]\n",
    "\n",
    "```"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
